# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import codecs
import json

import pymysql
from scrapy.pipelines.images import ImagesPipeline
from scrapy.exporters import JsonItemExporter
from twisted.enterprise import adbapi
from scrapy.utils.project import get_project_settings
from DataSpider.models.es_types import WuYictoIndex


class FirtscrapyPipeline(object):
    def process_item(self, item, spider):
        return item


class ArticleImagesPipline(ImagesPipeline):
    # 返回图片的路径
    def item_completed(self, results, item, info):
        if "front_image_url" in item:
            for ok, value in results:
                image_file_path = value["path"]
            item["front_image_path"] = image_file_path
        return item


class JsonWithEncodingPipeline(object):
    # 自定义的Json文件
    def __init__(self):
        self.file = codecs.open('article.json', 'w', 'utf-8')

    def process_item(self, item, spider):
        lines = json.dumps(dict(item), ensure_ascii=False) + "\n"
        self.file.write(lines)
        return item

    def spider_closed(self, spider):
        self.file.closed()


class JsonItemExporterPipeline(object):
    # 通过scrapy提供的Json export导出json文件
    def __init__(self):
        self.file = open('articleexport.json', 'wb')
        self.exporter = JsonItemExporter(self.file, encoding='utf-8', ensure_ascii=False)
        self.exporter.start_exporting()

    def closer_spider(self, spider):
        self.exporter.finish_exporting()
        self.file.close()

    def process_item(self, item, spider):
        self.exporter.export_item(item)
        return item


class MysqlPipeline(object):
    def __init__(self):
        self.conn = pymysql.connect('127.0.0.1', 'root', '123456', 'search_spider', charset='utf8', use_unicode=True)
        self.cursor = self.conn.cursor()

    def process_item(self, item, spider):
        insert_sql = """
         insert into 51cto_article (url_object_id, title, url, create_time, front_image_url, front_image_path, author, origin, origin_url,
            praise_num,  content, tags) 
            values ( %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
        """
        self.cursor.execute(insert_sql, (
            item["url_object_id"], item["title"], item["url"], item["create_time"], item["front_image_url"],
            item["front_image_path"], item["author"], item["origin"], item["origin_url"], item["praise_num"],
            item["content"], item["tags"]))
        self.conn.commit()


class MysqlTwistedPipeline(object):
    def __init__(self, dbpool):
        self.dbpool = dbpool

    @classmethod
    def from_crawler(cls, crawler):
        settings = get_project_settings()
        # host = settings.get("MYSQL_HOST")
        dbparms = dict(
            host=settings.get("MYSQL_HOST"),
            db=settings.get("MYSQL_DBNAME"),
            user=settings.get("MYSQL_USER"),
            passwd=settings.get("MYSQL_PASSWD"),
            charset='utf8',
            cursorclass=pymysql.cursors.DictCursor,
            use_unicode=True,
        )
        dbpool = adbapi.ConnectionPool("pymysql", **dbparms)
        return cls(dbpool)

    def process_item(self, item, spider):
        # 数据库异步插入
        query = self.dbpool.runInteraction(self.do_insert, item)
        query.addErrback(self.handle_error, item, spider)

    def handle_error(self, failure, item, spider):
        # 处理异步插入的异常
        print(failure)

    def do_insert(self, cursor, item):
        # 执行具体的插入
        # 根据不同的item 构建不同的sql语句并插入到mysql中
        insert_sql, params = item.get_insert_sql()
        cursor.execute(insert_sql, params)


class ElasticsearchPipline(object):
    # 将数据写入到es中
    def process_item(self, item, spider):
        # 将item转换为es数据
        item.save_to_es()
        return item
